# -*- coding: utf-8 -*-
# @Time    : 2020/3/17 10:36
# @Author  : ZSQ
# @Email   : zsq199170918@163.com
# @FileName: ganzhou.py
# @Software: PyCharm
import scrapy

import json
import logging
import demjson
from copy import copy, deepcopy
from FDC_spider.items import FdcEstateGuidItem, FdcBuildingItem, FdcRoomItem

logger = logging.getLogger(__name__)


class GanzhouSpider(scrapy.Spider):
    name = 'ganzhou'
    allowed_domains = ['218.64.195.220']
    start_urls = ['http://218.64.195.220:333/InternalWebForms/Item/lpbcx.aspx']
    company_li_url = 'http://218.64.195.220:333/jsonHandle/GetKFSItemTree_ForManager?bid=&fid=&nc='  # POST   开发商列表url
    rm_li_temp_url = 'http://218.64.195.220:333/JsonHandle/GetHouseResourcesData_House_DW?bid=&fid=&nc=&dbh={}&hid={}'  # POST   房号列表url

    def parse(self, response):
        h_id = response.xpath("//input[@id='h_id']/@value").extract_first()
        option_li = response.xpath("//select[@id='QYJG']/option[position()>1]")
        for index, option in enumerate(option_li):
            item_eg = FdcEstateGuidItem()
            item_eg['districtName'] = option.xpath("./text()").extract_first()
            option_value = option.xpath("./@value").extract_first()
            # 对章贡区进行特殊处理
            params_li = ['置业', '实业', '发展', '开发', '地产', '集团', '资产'] if not index else ['公司']
            for params in params_li:
                # 构造POST请求参数
                data = dict(
                    h_id=h_id,
                    QYJG=option_value,
                    type=str(0),
                    Accuracy=str(False),
                    condition='{}'.format(params)
                )
                yield scrapy.FormRequest(
                    self.company_li_url,
                    formdata=data,
                    callback=self.parse_company_li,
                    meta=dict(item_eg=deepcopy(item_eg), h_id=h_id, condition=deepcopy(data['condition'])),
                    dont_filter=True,
                )

    def parse_company_li(self, response):
        """
        获取开发商列表(包括项目列表和楼栋列表)
        :param response:
        :return:
        """
        item_eg = copy(response.meta['item_eg'])
        condition = copy(response.meta['condition'])
        h_id = copy(response.meta['h_id'])
        try:
            resp_dict = json.loads(response.body.decode(errors='ignore'))
            succeed = resp_dict['Succeed']
            try:
                data_dict = json.loads(self._json_propt(resp_dict['Data']))
            except:
                data_dict = demjson.decode(self._json_propt(resp_dict['Data']))
            item_li = data_dict['item']
            assert succeed and len(item_li)
        except Exception as e:
            logger.warning('{} 以 {} 作为查询条件，返回开发商列表为空或获取响应失败,error_msg:{}'.format(item_eg['districtName'], condition, e))
        else:
            for company_dict in item_li:
                item_eg['developerName'] = company_dict['text']
                prt_li = company_dict['item']
                for prt_dict in prt_li:
                    item_eg['projectName'] = prt_dict['text']
                    item_eg['projectId'] = prt_dict['id']
                    yield item_eg
                    # 获取楼栋列表
                    bd_li = prt_dict['item']
                    for bd_dict in bd_li:
                        item_bd = FdcBuildingItem()
                        item_bd['districtName'] = item_eg['districtName']
                        item_bd['developerName'] = item_eg['developerName']
                        item_bd['projectName'] = item_eg['projectName']
                        item_bd['projectId'] = item_eg['projectId']
                        item_bd['blockName'] = bd_dict['text']
                        try:
                            item_bd['buildingId'] = bd_dict['item'][0]['id']
                        except:
                            item_bd['buildingId'] = None
                            item_bd['floorTotalNo'] = None
                            yield item_bd
                        else:
                            # 构造POST请求，获取房号列表
                            headers = dict(referer=self.start_urls[0])
                            yield scrapy.Request(
                                self.rm_li_temp_url.format(item_bd['buildingId'], h_id),
                                method='POST',
                                headers=headers,
                                callback=self.parse_rm_li,
                                meta=dict(item_bd=deepcopy(item_bd)),
                                priority=10,
                            )

    def parse_rm_li(self, response):
        """
        获取房号列表
        :param response:
        :return:
        """
        item_bd = copy(response.meta['item_bd'])
        try:
            resp_dict = json.loads(response.body.decode(errors='ignore'))
            succeed = resp_dict['Succeed']
            data_li = resp_dict['Data']
            assert succeed and len(data_li)
        except:
            logger.error(
                "{}-{}-{}-{} 房号列表获取错误".format(item_bd['districtName'], item_bd['developerName'], item_bd['projectName'],
                                              item_bd['blockName']))
            item_bd['floorTotalNo'] = None
            yield item_bd
        else:
            for index, rm_dict in enumerate(data_li):
                if not index:
                    try:
                        item_bd['floorTotalNo'] = int(rm_dict.get('ZCS'))
                    except:
                        item_bd['floorTotalNo'] = rm_dict.get('ZCS', None)
                    finally:
                        yield item_bd
                item_rm = FdcRoomItem()
                item_rm['districtName'] = item_bd['districtName']
                item_rm['projectName'] = item_bd['projectName']
                item_rm['blockName'] = item_bd['blockName']
                item_rm['buildingId'] = item_bd['buildingId']
                item_rm['roomId'] = rm_dict.get('HouseID', None)
                item_rm['roomNo'] = rm_dict.get('HouseNo', None)
                item_rm['nominalFloor'] = rm_dict.get('MYC', None)
                item_rm['physicsFloor'] = rm_dict.get('SJC', None)
                item_rm['floorTotalNo'] = rm_dict.get('ZCS', None)
                item_rm['floorTotalNo'] = int(item_rm['floorTotalNo']) if isinstance(item_rm['floorTotalNo'],
                                                                                     float) else item_rm['floorTotalNo']
                item_rm['roomArea'] = rm_dict.get('BuildArea', None)
                item_rm['innerArea'] = rm_dict.get('InnerArea', None)
                item_rm['roomStructure'] = rm_dict.get('HouseStyle', None)
                item_rm['roomUse'] = rm_dict.get('FWYT_S', None)
                item_rm['saleStatus'] = rm_dict.get('FwztMC', None)
                yield item_rm

    @staticmethod
    def _json_propt(json_str):
        """
        处理非标准json，给json内的键加上双引号
        :param astr:
        :return:
        """
        return json_str.replace(' ', '').replace('\n', '').replace('\r', '').replace("'", '"').replace('{',
                                                                                                       '{"').replace(
            ':', '":').replace('],', '],"').replace('",', '","')